const axios = require("axios");
const path = require("path");
const crypto = require("crypto");
const fileUtil = require("@/utils/file.util");
const AppConfig = require("$config/app.config");
const tobaccoCrawlersModel = require("./tobacco-crawlers.model");
const arrayUtil = require("@/utils/array.util");
const { TenantContext } = require("@/core/tenant-context");
const ParserUtil = require("../helpers/parse.util");

/**
 * 烟草爬虫
 * @param {Object} config 配置信息
 * @param {string} config.baseUrl 基础地址
 * @param {string} config.listSelector 列表容器选择器
 * @param {number} config.delay 延时
 * @param {Object} config.selectors 所有属性选择器，参考
 */
class TobaccoCrawlersService {
	constructor(config) {
		this.config = {
			concurrency: 5,
			maxRequests: 100,
			delay: 1000,
			...config
		};
	}

	/**
	 * 爬虫
	 */
	async crawl() {
		const { config } = this;
		if (!config.baseUrl || !config.listSelector) {
			throw new Error("Missing required config: baseUrl and listSelector");
		}

		const results = [];
		let requestCount = 0;
		let pageCount = 1;
		let currentUrl = this.parseUrl(1);
		while (currentUrl && pageCount <= config.pager.maxPage && requestCount < config.maxRequests) {
			try {
				console.log(`Crawling page ${pageCount}: ${currentUrl}`);
				// 获取html页面
				const html = await this.fetchPage(currentUrl, config);
				if (!html) break;
				// 处理列表页
				requestCount++;
				const pageResults = ParserUtil.listParse(html, this.config);
				results.push(...pageResults);

				// 处理分页
				currentUrl = this.parseUrl(pageCount + 1);
				pageCount++;

				// 延迟防止请求频繁
				await this.delay();
			} catch (error) {
				console.error(`Error crawling ${currentUrl}:`, error.message);
				break;
			}
		}
		return results;
	}

	/**
	 * 格式化数据
	 */
	formatCrawlData(dataList, downloadList) {
		const downloadMap = downloadList.reduce((res, cur) => {
			res[cur.originalUrl] = cur;
			return res;
		}, {});
		return dataList.map(itm => {
			const { image } = itm;
			if (downloadMap[image] && downloadMap[image].success) {
				itm.image = downloadMap[image];
			}
			return itm;
		});
	}

	/**
	 * 写入数据库中
	 */
	async writeDatabases(dataList) {
		const chunks = arrayUtil.groupBySize(dataList, 20);
		const getFileDatabasePath = path => {
			return "/" + path.replace(/\\/g, "/").split("/").slice(-2).join("/");
		};
		// 写数据库方法
		const insertDB = dataList => {
			return new Promise(async resolve => {
				/**
				 * 执行步骤：
				 * 1、先将文件存入文件表中
				 * 2、将数据和文档表数据组合，写入素材表中
				 * 3、返回写入状态
				 */
				const sqlParams = [];
				for (let i = 0; i < dataList.length; i++) {
					const { image } = dataList[i];
					const md5 = await fileUtil.getFileMD5(image.localPath); // 设置文件hash值
					sqlParams.push({
						name: fileUtil.getFileNameByPath(image.localPath),
						original_name: fileUtil.getFileNameByPath(image.originalUrl),
						path: getFileDatabasePath(image.localPath),
						suffix: image.localPath.split(".").pop().toLowerCase(),
						size: image.size,
						md5_hash: md5,
						status: 0
					});
				}

				// 存储文件信息
				const [_fileError, _fileData] = await tobaccoCrawlersModel.createMultipleMaterialFile(sqlParams);
				if (_fileError) {
					console.log("烟草爬虫数据文件写入数据库失败：", _fileError);
					return;
				}

				// 创建素材数据
				// const [err, data] = await tobaccoCrawlersModel.createMaterial({
				//     name: '',
				//     image_id: '',
				//     category_id: '',
				//     size: ''
				// })
			});
		};
		// 将分组切片的文件写入数据库中
		for (let i = 0; i < chunks.length; i++) {
			let chunkItem = chunks[i];
			await insertDB(chunkItem);
		}
	}

	/**
	 * 延迟
	 * @returns
	 */
	delay() {
		return new Promise(resolve => setTimeout(resolve, this.config.delay));
	}

	/**
	 * 格式化url
	 * @param {*} pageIndex
	 * @returns
	 */
	parseUrl(pageIndex) {
		const baseUrl = this.config.baseUrl;
		const parseParams = this.config.parseUrlParams(pageIndex);
		if (baseUrl.indexOf("?") > -1) {
			if (baseUrl[baseUrl.length - 1] !== "&") {
				return `${baseUrl}&${parseParams}`;
			}
			return `${baseUrl}${parseParams}`;
		}
		return `${baseUrl}?${parseParams}`;
	}

	/**
	 * 请求页面
	 * @param {*} url
	 * @param {*} config
	 * @returns
	 */
	async fetchPage(url, config) {
		try {
			const response = await axios.get(url, {
				headers: {
					"User-Agent":
						config.userAgent ||
						"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
					...(config.headers || {})
				},
				timeout: config.timeout || 10000,
				...(config.axiosOptions || {})
			});
			return response.data;
		} catch (error) {
			console.error(`Error fetching ${url}:`, error.message);
			return null;
		}
	}

	/**
	 * 获取数据行hash
	 * @param {*} param0
	 * @returns
	 */
	_getRowHash({ name, brand_name, image_url, price }) {
		const str = [name, brand_name, image_url, price].join(".");
		const md5 = crypto.createHash("md5").update(str).digest("hex");
		return String(md5).toUpperCase();
	}

	/**
	 * 对比数据列表
	 * @param {*} dataList
	 * @param {*} originalDataList
	 * @returns
	 */
	_diffDataList(dataList, originalDataList) {
		const updates = [];
		const inserts = [];
		// 非空判断
		if (!originalDataList || originalDataList.length === 0) {
			return {
				updates: [],
				inserts: dataList
			};
		}

		// 源数据map
		const originalDataMap = originalDataList.reduce((result, cur) => {
			result[`${cur.name}-${cur.brand_name}`] = cur;
			return result;
		}, {});

		// 循环对比
		for (let i = 0, len = dataList.length; i < len; i++) {
			const item = dataList[i];
			const mapKey = `${item.name}-${item.brand_name}`;
			// 在源数据中不存在则为，新增数据
			if (!originalDataMap[mapKey]) {
				inserts.push(item);
			} else {
				// 如果在源数据中存在，则判断源数据的hash与新生成的hash是否一致
				if (item.hash !== originalDataMap[mapKey].hash) {
					updates.push({
						id: originalDataMap[mapKey].id,
						row: item
					});
				}
			}
		}
		return {
			updates,
			inserts
		};
	}

	/**
	 * 写入数据
	 */
	async writeTobaccoCrawlerData(data) {
		try {
			if (!Array.isArray(data)) return;
			// 格式化数据
			const crawlerData = data.map(item => {
				const rowHash = this._getRowHash({
					name: item.productName,
                    brand_name: item.name,
                    image_url: item.image,
                    price: item.price
				});
				return {
					name: item.productName,
					brand_name: item.name,
					production_address: "",
					type: 3,
					reference_price: item.price,
					image_url: item.image,
					hash: rowHash
				};
			});
			const databaseResult = await tobaccoCrawlersModel.queryTobaccoDataList();
			if (databaseResult.code !== 200 || !Array.isArray(databaseResult.results)) {
				console.log("获取数据库中烟草数据失败");
				return;
			}

			// 对比获取写入、修改的数据
			const { inserts, updates } = this._diffDataList(crawlerData, databaseResult.results);

			// 新增数据
			if (inserts && inserts.length > 0) {
				TenantContext.setCurrentUser("system");
				const [_error, _data] = await tobaccoCrawlersModel.insertTobaccoCrawlerData(inserts);
				if (_error) {
					console.log("爬取到的数据写入数据库失败：", _error);
					return;
				}
				console.log(`爬虫烟草数据成功写入：${inserts.length}条`);
			}

			// 需要修改的数据
            if (updates && updates.length > 0) {
                TenantContext.setCurrentUser("system");
                const updateValue = updates.map(itm => {
                    return {
                        id: itm.id,
                        ...itm.row
                    }
                })
                const updateResult = await tobaccoCrawlersModel.updateTobaccoData(updateValue)
                if (updateResult.code === 200) {
                    console.log(`爬虫烟草数据成功更新：${updates.length}条`);
                }
			}
		} catch (error) {
			console.log("写入数据库失败");
		}
	}
}

const t1 = new TobaccoCrawlersService({
	baseUrl: "https://www.yanjiuhome.com/xiangyansx/1-0-0",
	delay: 3 * 1000,
	listSelector: ".pro-library .pl-box",
	selectors: {
		productName: {
			selector: ".b-title",
			attr: "title"
		},
		image: {
			selector: itemEl => {
				const img = itemEl.find(".b-img img").attr("lazy");
				return `https:${img}`;
			}
		},
		price: {
			selector: itemEl => {
				return parseInt(itemEl.find(".p-p").text());
			}
		},
		name: ".p-name"
	},
	pager: {
		maxPage: 1
	},
	// 格式化分页参数
	parseUrlParams: pageIndex => {
		return `page=${pageIndex}`;
	}
});

t1.crawl()
	.then(async data => {
		t1.writeTobaccoCrawlerData(data);
		console.log("/************* 爬虫数据写入数据库成功 *************/");

		return;
		const downloadList = dataList.map(itm => {
			return {
				name: `image-${Date.now()}${Math.floor(Math.random() * 10000)}.${itm.image.split(".").pop()}`,
				url: itm.image
			};
		});
		// 今日日期文件夹
		const now = new Date();
		const dayDir = [now.getFullYear(), now.getMonth() + 1, now.getDate()].map(itm => (itm < 10 ? "0" + itm : itm)).join("");
		// 下载文件夹
		const downloadFolder = path.join(AppConfig.FILE_UPLOAD_PATH, dayDir);
		// 创建文件夹
		await fileUtil.createFolder(downloadFolder);
		// 开始下载
		const downloadResult = await fileUtil.downloadImages(downloadList, {
			downloadDir: downloadFolder
		});
		// 组合爬到的数据和下载文件数据
		const formatCrawlData = t1.formatCrawlData(dataList, downloadResult);
		console.log(formatCrawlData);
		// 写入数据库中
		t1.writeDatabases(formatCrawlData);
	})
	.catch(error => {
		console.log(error);
	});
// const html = `
//   <ul>
//     <li>Item 1<h3 class="title">A</h3></li>
//     <li>Item 2<h3 class="title">B</h3></li>
//     <li>Item 3<h3 class="title">C</h3></li>
//   </ul>
// `;

// const result = ParserUtil._parseListItem({
//     html: html
// }, {
//     customers: {
//         selector: function ($) {
//             const result = [];
//             result.push($('li').first().find('.title').text())
//             return result;
//         }
//     }
// });

// console.log(result);

module.exports = {
	TobaccoCrawlersService
};
